{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import tarfile\n",
    "\n",
    "import requests\n",
    "\n",
    "from pugnlp.futil import path_status, find_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# From the nlpia package for downloading data too big for the repo\n",
    "\n",
    "BIG_URLS = {\n",
    "    'w2v': (\n",
    "        'https://www.dropbox.com/s/965dir4dje0hfi4/GoogleNews-vectors-negative300.bin.gz?dl=1',\n",
    "        1647046227,\n",
    "    ),\n",
    "    'slang': (\n",
    "        'https://www.dropbox.com/s/43c22018fbfzypd/slang.csv.gz?dl=1',\n",
    "        117633024,\n",
    "    ),\n",
    "    'tweets': (\n",
    "        'https://www.dropbox.com/s/5gpb43c494mc8p0/tweets.csv.gz?dl=1',\n",
    "        311725313,\n",
    "    ),\n",
    "    'lsa_tweets': (\n",
    "        'https://www.dropbox.com/s/rpjt0d060t4n1mr/lsa_tweets_5589798_2003588x200.tar.gz?dl=1',\n",
    "        3112841563,  # 3112841312,\n",
    "    ),\n",
    "    'imdb': (\n",
    "        'https://www.dropbox.com/s/yviic64qv84x73j/aclImdb_v1.tar.gz?dl=1',\n",
    "        3112841563,  # 3112841312,\n",
    "    ),\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# These functions are part of the nlpia package which can be pip installed and run from there.\n",
    "def dropbox_basename(url):\n",
    "    filename = os.path.basename(url)\n",
    "    match = re.findall(r'\\?dl=[0-9]$', filename)\n",
    "    if match:\n",
    "        return filename[:-len(match[0])]\n",
    "    return filename\n",
    "\n",
    "def download_file(url, data_path='.', filename=None, size=None, chunk_size=4096, verbose=True):\n",
    "    \"\"\"Uses stream=True and a reasonable chunk size to be able to download large (GB) files over https\"\"\"\n",
    "    if filename is None:\n",
    "        filename = dropbox_basename(url)\n",
    "    file_path = os.path.join(data_path, filename)\n",
    "    if url.endswith('?dl=0'):\n",
    "        url = url[:-1] + '1'  # noninteractive download\n",
    "    if verbose:\n",
    "        tqdm_prog = tqdm\n",
    "        print('requesting URL: {}'.format(url))\n",
    "    else:\n",
    "        tqdm_prog = no_tqdm\n",
    "    r = requests.get(url, stream=True, allow_redirects=True)\n",
    "    size = r.headers.get('Content-Length', None) if size is None else size\n",
    "    print('remote size: {}'.format(size))\n",
    "\n",
    "    stat = path_status(file_path)\n",
    "    print('local size: {}'.format(stat.get('size', None)))\n",
    "    if stat['type'] == 'file' and stat['size'] == size:  # TODO: check md5 or get the right size of remote file\n",
    "        r.close()\n",
    "        return file_path\n",
    "\n",
    "    print('Downloading to {}'.format(file_path))\n",
    "\n",
    "    with open(file_path, 'wb') as f:\n",
    "        for chunk in r.iter_content(chunk_size=chunk_size):\n",
    "            if chunk:  # filter out keep-alive chunks\n",
    "                f.write(chunk)\n",
    "\n",
    "    r.close()\n",
    "    return file_path\n",
    "\n",
    "def untar(fname):\n",
    "    if fname.endswith(\"tar.gz\"):\n",
    "        with tarfile.open(fname) as tf:\n",
    "            tf.extractall()\n",
    "    else:\n",
    "        print(\"Not a tar.gz file: {}\".format(fname))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#  UNCOMMENT these 2 lines if you haven't already download the word2vec model and the imdb dataset\n",
    "# download_file(BIG_URLS['w2v'][0])\n",
    "# untar(download_file(BIG_URLS['imdb'][0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "maxlen = 400\n",
    "batch_size = 32\n",
    "embedding_dims = 300\n",
    "epochs = 2\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Flatten, LSTM\n",
    "\n",
    "num_neurons = 50\n",
    "\n",
    "print('Build model...')\n",
    "model = Sequential()\n",
    "\n",
    "model.add(LSTM(num_neurons, return_sequences=True, input_shape=(maxlen, embedding_dims)))\n",
    "model.add(Dropout(.2))\n",
    "\n",
    "model.add(Flatten())\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.compile('rmsprop', 'binary_crossentropy',  metrics=['accuracy'])\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import glob\n",
    "import os\n",
    "\n",
    "from random import shuffle\n",
    "\n",
    "def pre_process_data(filepath):\n",
    "    \"\"\"\n",
    "    This is dependent on your training data source but we will try to generalize it as best as possible.\n",
    "    \"\"\"\n",
    "    positive_path = os.path.join(filepath, 'pos')\n",
    "    negative_path = os.path.join(filepath, 'neg')\n",
    "    \n",
    "    pos_label = 1\n",
    "    neg_label = 0\n",
    "    \n",
    "    dataset = []\n",
    "    \n",
    "    for filename in glob.glob(os.path.join(positive_path, '*.txt')):\n",
    "        with open(filename, 'r') as f:\n",
    "            dataset.append((pos_label, f.read()))\n",
    "            \n",
    "    for filename in glob.glob(os.path.join(negative_path, '*.txt')):\n",
    "        with open(filename, 'r') as f:\n",
    "            dataset.append((neg_label, f.read()))\n",
    "    \n",
    "    shuffle(dataset)\n",
    "    \n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from nltk.tokenize import TreebankWordTokenizer\n",
    "from gensim.models import KeyedVectors\n",
    "word_vectors = KeyedVectors.load_word2vec_format('GoogleNews-vectors-negative300.bin.gz', binary=True, limit=200000)\n",
    "\n",
    "def tokenize_and_vectorize(dataset):\n",
    "    tokenizer = TreebankWordTokenizer()\n",
    "    vectorized_data = []\n",
    "    expected = []\n",
    "    for sample in dataset:\n",
    "        tokens = tokenizer.tokenize(sample[1])\n",
    "        sample_vecs = []\n",
    "        for token in tokens:\n",
    "            try:\n",
    "                sample_vecs.append(word_vectors[token])\n",
    "\n",
    "            except KeyError:\n",
    "                pass  # No matching token in the Google w2v vocab\n",
    "            \n",
    "        vectorized_data.append(sample_vecs)\n",
    "\n",
    "    return vectorized_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def collect_expected(dataset):\n",
    "    \"\"\" Peel of the target values from the dataset \"\"\"\n",
    "    expected = []\n",
    "    for sample in dataset:\n",
    "        expected.append(sample[0])\n",
    "    return expected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def pad_trunc(data, maxlen):\n",
    "    \"\"\" For a given dataset pad with zero vectors or truncate to maxlen \"\"\"\n",
    "    new_data = []\n",
    "\n",
    "    # Create a vector of 0's the length of our word vectors\n",
    "    zero_vector = []\n",
    "    for _ in range(len(data[0][0])):\n",
    "        zero_vector.append(0.0)\n",
    "\n",
    "    for sample in data:\n",
    " \n",
    "        if len(sample) > maxlen:\n",
    "            temp = sample[:maxlen]\n",
    "        elif len(sample) < maxlen:\n",
    "            temp = sample\n",
    "            additional_elems = maxlen - len(sample)\n",
    "            for _ in range(additional_elems):\n",
    "                temp.append(zero_vector)\n",
    "        else:\n",
    "            temp = sample\n",
    "        new_data.append(temp)\n",
    "    return new_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "dataset = pre_process_data('./aclImdb_v1/train')\n",
    "vectorized_data = tokenize_and_vectorize(dataset)\n",
    "expected = collect_expected(dataset)\n",
    "\n",
    "split_point = int(len(vectorized_data)*.8)\n",
    "\n",
    "x_train = vectorized_data[:split_point]\n",
    "y_train = expected[:split_point]\n",
    "x_test = vectorized_data[split_point:]\n",
    "y_test = expected[split_point:]\n",
    "\n",
    "maxlen = 400\n",
    "batch_size = 32         # How many samples to show the net before backpropogating the error and updating the weights\n",
    "embedding_dims = 300    # Length of the token vectors we will create for passing into the Convnet\n",
    "epochs = 2\n",
    "\n",
    "x_train = pad_trunc(x_train, maxlen)\n",
    "x_test = pad_trunc(x_test, maxlen)\n",
    "\n",
    "x_train = np.reshape(x_train, (len(x_train), maxlen, embedding_dims))\n",
    "y_train = np.array(y_train)\n",
    "x_test = np.reshape(x_test, (len(x_test), maxlen, embedding_dims))\n",
    "y_test = np.array(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Flatten, LSTM\n",
    "\n",
    "num_neurons = 50\n",
    "\n",
    "print('Build model...')\n",
    "model = Sequential()\n",
    "\n",
    "model.add(LSTM(num_neurons, return_sequences=True, input_shape=(maxlen, embedding_dims)))\n",
    "model.add(Dropout(.2))\n",
    "\n",
    "model.add(Flatten())\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.compile('rmsprop', 'binary_crossentropy',  metrics=['accuracy'])\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.fit(x_train, y_train,\n",
    "          batch_size=batch_size,\n",
    "          epochs=epochs,\n",
    "          validation_data=(x_test, y_test))\n",
    "model_structure = model.to_json()\n",
    "with open(\"lstm_model1.json\", \"w\") as json_file:\n",
    "    json_file.write(model_structure)\n",
    "\n",
    "model.save_weights(\"lstm_weights1.h5\")\n",
    "print('Model saved.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.models import model_from_json\n",
    "with open(\"lstm_model1.json\", \"r\") as json_file:\n",
    "    json_string = json_file.read()\n",
    "model = model_from_json(json_string)\n",
    "\n",
    "model.load_weights('lstm_weights1.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sample_1 = \"I'm hate that the dismal weather that had me down for so long, when will it break! Ugh, when does happiness return?  The sun is blinding and the puffy clouds are too thin.  I can't wait for the weekend.\"\n",
    "\n",
    "# We pass a dummy value in the first element of the tuple just because our helper expects it from the way processed the initial data.  That value won't ever see the network, so it can be whatever.\n",
    "vec_list = tokenize_and_vectorize([(1, sample_1)])\n",
    "\n",
    "# Tokenize returns a list of the data (length 1 here)\n",
    "test_vec_list = pad_trunc(vec_list, maxlen)\n",
    "\n",
    "test_vec = np.reshape(test_vec_list, (len(test_vec_list), maxlen, embedding_dims))\n",
    "\n",
    "print(\"Sample's sentiment, 1 - pos, 2 - neg : {}\".format(model.predict_classes(test_vec)))\n",
    "print(\"Raw output of sigmoid function: {}\".format(model.predict(test_vec)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def test_len(data, maxlen):\n",
    "    total_len = truncated = exact = padded = 0\n",
    "    for sample in data:\n",
    "        total_len += len(sample)\n",
    "        if len(sample) > maxlen:\n",
    "            truncated += 1\n",
    "        elif len(sample) < maxlen:\n",
    "            padded += 1\n",
    "        else:\n",
    "            exact +=1 \n",
    "    print('Padded: {}'.format(padded))\n",
    "    print('Equal: {}'.format(exact))\n",
    "    print('Truncated: {}'.format(truncated))\n",
    "    print('Avg length: {}'.format(total_len/len(data)))\n",
    "\n",
    "dataset = pre_process_data('./aclImdb_v1/train')\n",
    "vectorized_data = tokenize_and_vectorize(dataset)\n",
    "test_len(vectorized_data, 400)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Flatten, LSTM\n",
    "\n",
    "\n",
    "maxlen = 200\n",
    "batch_size = 32         # How many samples to show the net before backpropagating the error and updating the weights\n",
    "embedding_dims = 300    # Length of the token vectors we will create for passing into the Convnet\n",
    "\n",
    "epochs = 2\n",
    "\n",
    "dataset = pre_process_data('./aclImdb_v1/train')\n",
    "vectorized_data = tokenize_and_vectorize(dataset)\n",
    "expected = collect_expected(dataset)\n",
    "\n",
    "split_point = int(len(vectorized_data)*.8)\n",
    "\n",
    "x_train = vectorized_data[:split_point]\n",
    "y_train = expected[:split_point]\n",
    "x_test = vectorized_data[split_point:]\n",
    "y_test = expected[split_point:]\n",
    "\n",
    "x_train = pad_trunc(x_train, maxlen)\n",
    "x_test = pad_trunc(x_test, maxlen)\n",
    "\n",
    "x_train = np.reshape(x_train, (len(x_train), maxlen, embedding_dims))\n",
    "y_train = np.array(y_train)\n",
    "x_test = np.reshape(x_test, (len(x_test), maxlen, embedding_dims))\n",
    "y_test = np.array(y_test)\n",
    "\n",
    "num_neurons = 50\n",
    "\n",
    "print('Build model...')\n",
    "model = Sequential()\n",
    "\n",
    "model.add(LSTM(num_neurons, return_sequences=True, input_shape=(maxlen, embedding_dims)))\n",
    "model.add(Dropout(.2))\n",
    "\n",
    "model.add(Flatten())\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.compile('rmsprop', 'binary_crossentropy',  metrics=['accuracy'])\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.fit(x_train, y_train,\n",
    "          batch_size=batch_size,\n",
    "          epochs=epochs,\n",
    "          validation_data=(x_test, y_test))\n",
    "model_structure = model.to_json()\n",
    "with open(\"lstm_model7.json\", \"w\") as json_file:\n",
    "    json_file.write(model_structure)\n",
    "\n",
    "model.save_weights(\"lstm_weights7.h5\")\n",
    "print('Model saved.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dataset = pre_process_data('./aclImdb_v1/train')\n",
    "expected = collect_expected(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def avg_len(data):\n",
    "    total_len = 0\n",
    "    for sample in data:\n",
    "        total_len += len(sample[1])\n",
    "    print(total_len/len(data))\n",
    "\n",
    "print(avg_len(dataset))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def clean_data(data):\n",
    "    \"\"\" Shift to lower case, replace unknowns with UNK, and listify \"\"\"\n",
    "    new_data = []\n",
    "    VALID = 'abcdefghijklmnopqrstuvwxyz123456789\"\\'?!.,:; '\n",
    "    for sample in data:\n",
    "        new_sample = []\n",
    "        for char in sample[1].lower():  # Just grab the string, not the label\n",
    "            if char in VALID:\n",
    "                new_sample.append(char)\n",
    "            else:\n",
    "                new_sample.append('UNK')\n",
    "       \n",
    "        new_data.append(new_sample)\n",
    "    return new_data\n",
    "\n",
    "listified_data = clean_data(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def char_pad_trunc(data, maxlen):\n",
    "    \"\"\" We truncate to maxlen or add in PAD tokens \"\"\"\n",
    "    new_dataset = []\n",
    "    for sample in data:\n",
    "        if len(sample) > maxlen:\n",
    "            new_data = sample[:maxlen]\n",
    "        elif len(sample) < maxlen:\n",
    "            pads = maxlen - len(sample)\n",
    "            new_data = sample + ['PAD'] * pads\n",
    "        else:\n",
    "            new_data = sample\n",
    "        new_dataset.append(new_data)\n",
    "    return new_dataset\n",
    "\n",
    "maxlen = 1500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def create_dicts(data):\n",
    "    \"\"\" Modified from Keras LSTM example\"\"\"\n",
    "    chars = set()\n",
    "    for sample in data:\n",
    "        chars.update(set(sample))\n",
    "    char_indices = dict((c, i) for i, c in enumerate(chars))\n",
    "    indices_char = dict((i, c) for i, c in enumerate(chars))\n",
    "    return char_indices, indices_char"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "def onehot_encode(dataset, char_indices, maxlen):\n",
    "    \"\"\" \n",
    "    One hot encode the tokens\n",
    "    \n",
    "    Args:\n",
    "        dataset  list of lists of tokens\n",
    "        char_indices  dictionary of {key=character, value=index to use encoding vector}\n",
    "        maxlen  int  Length of each sample\n",
    "    Return:\n",
    "        np array of shape (samples, tokens, encoding length)\n",
    "    \"\"\"\n",
    "    X = np.zeros((len(dataset), maxlen, len(char_indices.keys())))\n",
    "    for i, sentence in enumerate(dataset):\n",
    "        for t, char in enumerate(sentence):\n",
    "            X[i, t, char_indices[char]] = 1\n",
    "    return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dataset = pre_process_data('./aclImdb_v1/train')\n",
    "expected = collect_expected(dataset)\n",
    "listified_data = clean_data(dataset)\n",
    "\n",
    "maxlen = 1500\n",
    "common_length_data = char_pad_trunc(listified_data, maxlen)\n",
    "\n",
    "char_indices, indices_char = create_dicts(common_length_data)\n",
    "encoded_data = onehot_encode(common_length_data, char_indices, maxlen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "split_point = int(len(encoded_data)*.8)\n",
    "\n",
    "x_train = encoded_data[:split_point]\n",
    "y_train = expected[:split_point]\n",
    "x_test = encoded_data[split_point:]\n",
    "y_test = expected[split_point:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Embedding, Flatten, LSTM\n",
    "\n",
    "\n",
    "num_neurons = 40\n",
    "\n",
    "print('Build model...')\n",
    "model = Sequential()\n",
    "\n",
    "model.add(LSTM(num_neurons, return_sequences=True, input_shape=(maxlen, len(char_indices.keys()))))\n",
    "model.add(Dropout(.2))\n",
    "\n",
    "model.add(Flatten())\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.compile('rmsprop', 'binary_crossentropy',  metrics=['accuracy'])\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch_size = 32\n",
    "epochs = 10\n",
    "model.fit(x_train, y_train,\n",
    "          batch_size=batch_size,\n",
    "          epochs=epochs,\n",
    "          validation_data=(x_test, y_test))\n",
    "model_structure = model.to_json()\n",
    "with open(\"char_lstm_model3.json\", \"w\") as json_file:\n",
    "    json_file.write(model_structure)\n",
    "\n",
    "model.save_weights(\"char_lstm_weights3.h5\")\n",
    "print('Model saved.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from nltk.corpus import gutenberg\n",
    "\n",
    "print(gutenberg.fileids())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "text = ''\n",
    "for txt in gutenberg.fileids():\n",
    "    if 'shakespeare' in txt:\n",
    "        text += gutenberg.raw(txt).lower()\n",
    "\n",
    "print('corpus length:', len(text))\n",
    "\n",
    "chars = sorted(list(set(text)))\n",
    "print('total chars:', len(chars))\n",
    "char_indices = dict((c, i) for i, c in enumerate(chars))\n",
    "indices_char = dict((i, c) for i, c in enumerate(chars))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print(text[:500])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# cut the text in semi-redundant sequences of maxlen characters\n",
    "maxlen = 40\n",
    "step = 3\n",
    "sentences = []\n",
    "next_chars = []\n",
    "for i in range(0, len(text) - maxlen, step):\n",
    "    sentences.append(text[i: i + maxlen])\n",
    "    next_chars.append(text[i + maxlen])\n",
    "print('nb sequences:', len(sentences))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print('Vectorization...')\n",
    "X = np.zeros((len(sentences), maxlen, len(chars)), dtype=np.bool)\n",
    "y = np.zeros((len(sentences), len(chars)), dtype=np.bool)\n",
    "for i, sentence in enumerate(sentences):\n",
    "    for t, char in enumerate(sentence):\n",
    "        X[i, t, char_indices[char]] = 1\n",
    "    y[i, char_indices[next_chars[i]]] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Activation\n",
    "from keras.layers import LSTM\n",
    "from keras.optimizers import RMSprop\n",
    "\n",
    "# build the model: a single LSTM\n",
    "print('Build model...')\n",
    "model = Sequential()\n",
    "model.add(LSTM(128, input_shape=(maxlen, len(chars))))\n",
    "model.add(Dense(len(chars)))\n",
    "model.add(Activation('softmax'))\n",
    "\n",
    "optimizer = RMSprop(lr=0.01)\n",
    "model.compile(loss='categorical_crossentropy', optimizer=optimizer)\n",
    "\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "epochs = 6\n",
    "batch_size = 128\n",
    "\n",
    "model_structure = model.to_json()\n",
    "with open(\"shakes_lstm_model.json\", \"w\") as json_file:\n",
    "    json_file.write(model_structure)\n",
    "\n",
    "for i in range(5):\n",
    "    model.fit(X, y,\n",
    "              batch_size=batch_size,\n",
    "              epochs=epochs)\n",
    "\n",
    "    model.save_weights(\"shakes_lstm_weights_{}.h5\".format(i+1))\n",
    "    print('Model saved.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### NOT IN CHAPTER, Just to reproduce output\n",
    "\n",
    "from keras.models import model_from_json\n",
    "\n",
    "with open('shakes_lstm_model.json', 'r') as f:\n",
    "    model_json = f.read()\n",
    "    \n",
    "model = model_from_json(model_json)\n",
    "model.load_weights('shakes_lstm_weights_4.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "def sample(preds, temperature=1.0):\n",
    "    # helper function to sample an index from a probability array\n",
    "    preds = np.asarray(preds).astype('float64')\n",
    "    preds = np.log(preds) / temperature\n",
    "    exp_preds = np.exp(preds)\n",
    "    preds = exp_preds / np.sum(exp_preds)\n",
    "    probas = np.random.multinomial(1, preds, 1)\n",
    "    return np.argmax(probas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "start_index = random.randint(0, len(text) - maxlen - 1)\n",
    "\n",
    "for diversity in [0.2, 0.5, 1.0]:\n",
    "    print()\n",
    "    print('----- diversity:', diversity)\n",
    "\n",
    "    generated = ''\n",
    "    sentence = text[start_index: start_index + maxlen]\n",
    "    generated += sentence\n",
    "    print('----- Generating with seed: \"' + sentence + '\"')\n",
    "    sys.stdout.write(generated)\n",
    "\n",
    "    for i in range(400):\n",
    "        x = np.zeros((1, maxlen, len(chars)))\n",
    "        for t, char in enumerate(sentence):\n",
    "            x[0, t, char_indices[char]] = 1.\n",
    "\n",
    "        preds = model.predict(x, verbose=0)[0]\n",
    "        next_index = sample(preds, diversity)\n",
    "        next_char = indices_char[next_index]\n",
    "\n",
    "        generated += next_char\n",
    "        sentence = sentence[1:] + next_char\n",
    "\n",
    "        sys.stdout.write(next_char)\n",
    "        sys.stdout.flush()\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import GRU\n",
    "\n",
    "model = Sequential()\n",
    "model.add(GRU(num_neurons, return_sequences=True, input_shape=X[0].shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import LSTM\n",
    "\n",
    "model = Sequential()\n",
    "model.add(LSTM(num_neurons, return_sequences=True, input_shape=X[0].shape))\n",
    "model.add(LSTM(num_neurons_2, return_sequences=True))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
